package com.gmss.crawler.service;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.gmss.crawler.constants.Constants;
import com.gmss.crawler.enums.AcademicianEnum;
import com.gmss.crawler.module.paper.domain.Paper;
import com.gmss.crawler.module.person.domain.PersonInfo;
import com.gmss.crawler.module.resume.domain.Resume;
import com.gmss.crawler.utils.CommonUtils;
import com.gmss.crawler.utils.CrawlerCommonUtils;
import com.gmss.crawler.utils.OkHttp3Util;
import com.gmss.crawler.utils.RegexUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.ClassPathResource;
import org.springframework.stereotype.Service;

import javax.annotation.Resource;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

/**
 * 国家杰出青年科学基金获得者
 * 简称：杰青
 */
@Slf4j
@Service
public class JQCrawlerService {

    @Value("${img_file_path}")
    private String imgFilePath;

    @Resource
    private ScholarService scholarService;

    /**
     * 两院院士中的国家杰出青年科学基金获得者，1994-2013
     *
     * @throws IOException
     */
    private static Map<String, Map<String, List<String>>> getAcademicianByOutstanTalents() throws IOException {
        org.springframework.core.io.Resource resource = new ClassPathResource("/htmlTest/jq-杰出青年/1994-2013两院院士中的国家杰出青年科学基金获得者.txt");
        String filePath = resource.getFile().getAbsolutePath();
        File file = new File(filePath);
        List<String> lineList = FileUtils.readLines(file, StandardCharsets.UTF_8);
        Map<String, Map<String, List<String>>> yearMap = new HashMap<>();
        for (int i = 0; i < lineList.size(); i++) {

            String year = lineList.get(i).split("：")[0];
            String[] lines = lineList.get(i).split(";");
            // 科学院
            List<String> casList = new ArrayList<>();
            // 工程院
            List<String> caeList = new ArrayList<>();
            // 其他，遗漏的
            List<String> otherList = new ArrayList<>();
            for (String line : lines) {
                if (StringUtils.isNotBlank(line)) {
                    line = line.replaceAll("（", "(").replaceAll("）", ")");
                    String academicianType = RegexUtils.getTextContent(line, RegexUtils.ALL_KUA_REGEX_TWO);
                    if (StringUtils.contains(academicianType, "科学院")) {
                        line = RegexUtils.replaceData(line, RegexUtils.ALL_KUA_REGEX, "");
                        line = line.split("，")[1];
                        String[] acads = line.split("、");
                        for (String str : acads) {
                            casList.add(str);
                        }

                    } else if (StringUtils.contains(academicianType, "工程院")) {
                        line = RegexUtils.replaceData(line, RegexUtils.ALL_KUA_REGEX, "");
                        String[] acads = line.split("、");
                        for (String str : acads) {
                            caeList.add(str);
                        }
                    } else {
                        if (line.indexOf("：") > 0) {
                            String acads = line.split("：")[1];
                            if (acads.indexOf("、") > 0) {
                                String[] acadArrays = acads.split("、");
                                for (String str : acadArrays) {
                                    otherList.add(str);
                                }
                            } else {
                                otherList.add(acads);
                            }
                        } else if (line.indexOf("、") > 0) {
                            String[] acads = line.split("、");
                            for (String str : acads) {
                                otherList.add(str);
                            }
                        } else {
                            otherList.add(line);
                        }
                    }
                }

                Map<String, List<String>> map = new HashMap<>();
                // 科学院
                map.put("cas", casList);
                // 工程院
                map.put("cae", caeList);
                if (CollectionUtils.isNotEmpty(otherList)) {
                    map.put("other", otherList);
                }
                if (StringUtils.isNotBlank(year))
                    yearMap.put(year, map);
            }
        }
        return yearMap;
    }

    /**
     * 封装1994-2019年杰出青年数据
     *
     * @return
     */
    public List<PersonInfo> crawlerOutstanTalents() {
        Map<String, Map<String, List<String>>> yearMap = new HashMap<>();
        // 已当选院士的杰青
        List<String> nameList = new ArrayList<>();
        try {
            yearMap = getAcademicianByOutstanTalents();
            if (MapUtils.isNotEmpty(yearMap)) {
                yearMap.forEach((key, value) -> {
                    if (MapUtils.isNotEmpty(value)) {
                        value.forEach((key1, value1) -> {
                            nameList.addAll(value1);
                        });
                    }
                });
            }
        } catch (IOException e) {
            log.error("1994-2013两院院士中的国家杰出青年科学基金获得者，Exceptoin：{}", e);
        }

//        Document doc = CrawlerCommonUtils.getDocumentHtml("/htmlTest/jq-杰出青年/1994-2018国家杰青.html");
        Document doc = CrawlerCommonUtils.getDocument("https://www.sciping.com/22844.html");
        Elements trElements = doc.select(".section-content tbody tr");
        List<PersonInfo> list = new ArrayList<>();
        PersonInfo personInfo = new PersonInfo();
        for (int i = 1; i < trElements.size(); i++) {
            Elements td = trElements.get(i).select("td");
            personInfo = new PersonInfo();
            String name = td.get(1).text();
            if (StringUtils.equals(name, "徐世娘")) {
                name = "徐世烺";
            }
            personInfo.setName(name);
            String text = td.get(2).text();
            int electionYear = Integer.parseInt(text);
            personInfo.setElectionYear(String.valueOf(electionYear));
            personInfo.setAcademicianTitle(AcademicianEnum.OUTSTANDING_YOUTH.getCode());
            personInfo.setAcademicianTitleValue(AcademicianEnum.OUTSTANDING_YOUTH.getValue());
            personInfo.setTitleYears(4);
            if (nameList.contains(name)) {
                personInfo.setStamp(1);
            }
            list.add(personInfo);
        }

        Elements tableElements = doc.select(".single-content table");
        Elements trEles = tableElements.get(2).select("tbody tr");
        for (int i = 1; i < trEles.size(); i++) {
            Elements td = trEles.get(i).select("td");
            personInfo = new PersonInfo();
            String name = td.get(1).text();
            personInfo.setName(name);
            personInfo.setWorkCompany(td.get(2).text());
            personInfo.setMajor(td.get(3).text());
            String text = td.get(4).text();
            int electionYear = Integer.parseInt(text);
            personInfo.setElectionYear(electionYear + "");
            personInfo.setAcademicianTitle(AcademicianEnum.OUTSTANDING_YOUTH.getCode());
            personInfo.setAcademicianTitleValue(AcademicianEnum.OUTSTANDING_YOUTH.getValue());
            personInfo.setTitleYears(4);
            if (nameList.contains(name)) {
                personInfo.setStamp(1);
            }
            list.add(personInfo);
        }

//        Document document = CrawlerCommonUtils.getDocumentHtml("/htmlTest/jq-杰出青年/1997-2006国家杰青.html");
        Document document = CrawlerCommonUtils.getDocument("https://www.sciping.com/23054.html");
        Elements trs = document.select(".single-content table tbody tr");
        for (int i = 1; i < trs.size(); i++) {
            Elements td = trs.get(i).select("td");
            personInfo = new PersonInfo();
            String name = td.get(1).text();
            personInfo.setName(name);
            personInfo.setWorkCompany(td.get(2).text());
            personInfo.setMajor(td.get(3).text());
            String text = td.get(4).text();
            int electionYear = Integer.parseInt(text);
            personInfo.setElectionYear(electionYear + "");
            personInfo.setAcademicianTitle(AcademicianEnum.OUTSTANDING_YOUTH.getCode());
            personInfo.setAcademicianTitleValue(AcademicianEnum.OUTSTANDING_YOUTH.getValue());
            personInfo.setTitleYears(4);
            if (nameList.contains(name)) {
                personInfo.setStamp(1);
            }
            list.add(personInfo);
        }

        Document doc1 = CrawlerCommonUtils.getDocumentHtml("/htmlTest/jq-杰出青年/2019-杰出青年.html");
        Elements tr = doc1.select("#modellist-1618322 .mod_body .mod_cont table tbody tr");
        for (int i = 1; i < tr.size(); i++) {
            Elements td = tr.get(i).select("td");
            personInfo = new PersonInfo();
            String name = td.get(1).text();
            personInfo.setName(name);
            personInfo.setSex(td.get(2).text());
            personInfo.setAcademicDegree(td.get(3).text());
            personInfo.setPersonTitle(td.get(4).text());
            personInfo.setMajor(td.get(5).text());
            personInfo.setWorkCompany(td.get(6).text());
            personInfo.setAcademicianTitle(AcademicianEnum.OUTSTANDING_YOUTH.getCode());
            personInfo.setAcademicianTitleValue(AcademicianEnum.OUTSTANDING_YOUTH.getValue());
            personInfo.setTitleYears(4);
            personInfo.setElectionYear("2019");
            if (nameList.contains(name)) {
                personInfo.setStamp(1);
            }
            list.add(personInfo);
        }
        return list;
    }

    /**
     * 全球华人专家库-人员信息列表
     *
     * @param paramsMap 请求URL的入参,offset: (第几页 * size)，size：每页显示条数，eb：人员信息列表，query：按姓名查询
     * @param nameMap   人员信息
     */
    public List<String> aminerPersonInfoList(Map<String, Object> paramsMap, Map<String, PersonInfo> nameMap) {
        String requestPath = "[{\"action\":\"expertbase.search.gctsearch\",\"parameters\":{\"searchType\":\"ToBPerson\"," +
                "\"query\": \"$${0}\"," +
                "\"offset\":$${1}," +
                "\"size\":$${2}," +
                "\"aggregation\":[\"gender\",\"h_index\",\"nation\",\"lang\"],\"switches\":[\"lang_zh\",\"intell_expand\"]," +
                "\"filters\":{\"dims\":{\"eb\":[\"$${3}\"]}}}," +
                "\"schema\":{\"person\":[\"id\",\"name\",\"title\",\"name_zh\",\"avatar\",\"tags\"," +
                "{\"profile\":[\"position\",\"position_zh\",\"affiliation\",\"affiliation_zh\",\"org\",\"org_zh\",\"gender\"]}," +
                "{\"indices\":[\"hindex\",\"gindex\",\"pubs\",\"citations\",\"newStar\",\"risingStar\",\"activity\",\"diversity\",\"sociability\"]},\"tags_translated_zh\"]}}]";
//        Map<String, String> resultMap = new LinkedHashMap<>();
        List<String> resultList = new ArrayList<>();
        try {
            String year = paramsMap.containsKey("year") ? paramsMap.get("year").toString() : null;
            // 查询条件 ： 姓名
            String queryByName = paramsMap.containsKey("queryByName") ? paramsMap.get("queryByName").toString() : "";
            String aminerrequestPath = RegexUtils.replaceData(requestPath, "\\$\\$\\{0}", queryByName);
            aminerrequestPath = RegexUtils.replaceData(aminerrequestPath, "\\$\\$\\{1}", paramsMap.get("pageOffset").toString());
            aminerrequestPath = RegexUtils.replaceData(aminerrequestPath, "\\$\\$\\{2}", paramsMap.get("pageSize").toString());
            // 查询条件 ： 列表id
            aminerrequestPath = RegexUtils.replaceData(aminerrequestPath, "\\$\\$\\{3}", paramsMap.get("listId").toString());
            Thread.sleep(CommonUtils.randomRange(1000, 5000));
            // 第一页数据
            okhttp3.Response response = OkHttp3Util.doPostJSON(Constants.AMINER_JQ_PERSONS_LIST_PATH, aminerrequestPath, 5000, 3, false);
            String result = response.body().string();

//            okhttp3.Response response = null;
//            String result = "{\"data\":[{\"aggregation\":[{\"items\":[{\"count\":1,\"term\":\"Chinese\"}],\"name\":\"lang\"},{\"items\":[{\"count\":0,\"term\":\"*-10.0\",\"to\":10},{\"count\":0,\"from\":10,\"term\":\"10.0-20.0\",\"to\":20},{\"count\":1,\"from\":20,\"term\":\"20.0-30.0\",\"to\":30},{\"count\":0,\"from\":30,\"term\":\"30.0-40.0\",\"to\":40},{\"count\":0,\"from\":40,\"term\":\"40.0-50.0\",\"to\":50},{\"count\":0,\"from\":50,\"term\":\"50.0-60.0\",\"to\":60},{\"count\":0,\"from\":60,\"term\":\"60.0-*\"}],\"name\":\"h_index\"},{\"items\":[{\"count\":1,\"term\":\"Male\"}],\"name\":\"gender\"},{\"items\":[{\"count\":1,\"term\":\"China\"}],\"name\":\"nation\"}],\"context\":\"F5DEF4ED\",\"duration\":\"1.181750174s\",\"intellResults\":{},\"items\":[{\"avatar\":\"https://static.aminer.cn/upload/avatar/1354/149/992/562097d745ceff436be210eb.jpg\",\"id\":\"562097d745ceff436be210eb\",\"indices\":{\"activity\":2.7212,\"citations\":2630,\"diversity\":0.7293,\"gindex\":50,\"hindex\":27,\"newStar\":8.6606,\"pubs\":179,\"risingStar\":8.6606,\"sociability\":6.6883},\"name\":\"Yanan Du\",\"name_zh\":\"杜亚楠\",\"profile\":{\"affiliation\":\"School of Medicine Tsinghua University/Beijing CytoNiche Biotech Ltd.\",\"affiliation_zh\":\"清华大学医学院/北京华龛生物科技有限公司\",\"gender\":\"male\",\"position\":\"Research\",\"position_zh\":\"研究员\"},\"score\":382.48563,\"sourcetype\":\"person\",\"tags\":[\"Tissue Engineering\",\"Composite Material\",\"Hydrogel\",\"Microfluidics\",\"Material Properties\",\"Bottom Up\",\"Convection\",\"Directed Assembly\",\"High Throughput\",\"Polarity\",\"Biomedical Research\",\"Fluorescence\",\"Surface Tension\",\"Mechanobiology\",\"Computer Simulation\",\"Bioinformatics\"],\"tags_score\":[7,6,6,6,5,4,3,3,3,3,3,3,3,3,3,3],\"tags_translated_zh\":[\"组织工程\",\"复合材料\",\"水凝胶\",\"微流体\",\"材料性能\",\"自下而上\",\"对流\",\"\",\"高通量\",\"极性\",\"生物医学研究\",\"荧光\",\"表面张力\",\"力学生物学\",\"计算机模拟\",\"生物信息学\"]}],\"keyValues\":{\"topics\":[null]},\"offset\":0,\"size\":20,\"succeed\":true,\"suggest\":null,\"total\":1}]}";
            if (StringUtils.isBlank(result))
                return resultList;

            result = CommonUtils.jsonUnicodeToUtf8(result);

            if (StringUtils.isNotBlank(result)) {
                resultList.add(result);
                JSONObject resultJson = JSON.parseObject(result);
                if (resultJson.containsKey("data") && resultJson.getJSONArray("data") != null && resultJson.getJSONArray("data").size() > 0) {
                    JSONArray dataArray = resultJson.getJSONArray("data");
                    JSONObject jsonObject = dataArray.getJSONObject(0);
                    if (null != jsonObject && jsonObject.getBoolean("succeed") && jsonObject.getInteger("total") > 0) {
                        // 全球华人专家库-杰青年度人员信息列表-解析
//                        parseItems(nameMap, jsonObject, year);

                        Integer offset = jsonObject.getInteger("offset");
                        Integer size = jsonObject.getInteger("size");
                        // 总条数
                        Integer total = jsonObject.getInteger("total");
                        // 计算总页数
                        Integer allPageSize = CommonUtils.getPages(total, size);

                        // 查询分页数据，从第二页开始
                        for (int i = 2; i <= allPageSize; i++) {
                            Integer newPageOffset = size + offset;
                            aminerrequestPath = RegexUtils.replaceData(requestPath, "\\$\\$\\{0}", paramsMap.containsKey("queryByName") ? paramsMap.get("queryByName").toString() : "");
                            aminerrequestPath = RegexUtils.replaceData(aminerrequestPath, "\\$\\$\\{1}", String.valueOf(newPageOffset));
                            aminerrequestPath = RegexUtils.replaceData(aminerrequestPath, "\\$\\$\\{2}", paramsMap.get("pageSize").toString());
                            aminerrequestPath = RegexUtils.replaceData(aminerrequestPath, "\\$\\$\\{3}", paramsMap.get("listId").toString());

//                            Thread.sleep(CommonUtils.randomRange(1000, 5000));
                            Thread.sleep(CommonUtils.randomRange(1000 * 60 * CommonUtils.randomRange(1, 3), 5000 * 60 * 1));
                            response = OkHttp3Util.doPostJSON(Constants.AMINER_JQ_PERSONS_LIST_PATH, aminerrequestPath, 5000, 3, false);
                            result = response.body().string();
                            if (StringUtils.isBlank(result))
                                continue;
                            result = CommonUtils.jsonUnicodeToUtf8(result);
                            resultList.add(result);

                            resultJson = JSON.parseObject(result);
                            if (resultJson.containsKey("data") && resultJson.getJSONArray("data") != null && resultJson.getJSONArray("data").size() > 0) {
                                dataArray = resultJson.getJSONArray("data");
                                jsonObject = dataArray.getJSONObject(0);
                                if (null != jsonObject && jsonObject.getBoolean("succeed") && jsonObject.getInteger("total") > 0) {
                                    offset = jsonObject.getInteger("offset");
                                    // 全球华人专家库-杰青年度人员信息列表-解析
//                                    parseItems(nameMap, jsonObject, year);
                                }
                            }
                        }
                    }
                }
            }
        } catch (IOException e) {
            log.error("全球华人专家库-杰青年度人员信息列表 - aminerPersonInfoList,Exception：{}", e);
        } catch (InterruptedException e) {
            log.error("全球华人专家库-杰青年度人员信息列表 - aminerPersonInfoList, Exception：{}", e);
        }
        return resultList;
    }


    /**
     * aminer-个人信息查询
     */
    public void aminerPersonInfoData(String queryPersonIds, Map<String, PersonInfo> aminerPersonMap) {
        String requestPathStr = "[{\"action\":\"personapi.get\",\"parameters\":{" +
//                "\"ids\":[\"54095308dabfae450f477709\"]}," +
                "\"ids\":[" + queryPersonIds + "]}," +
                "\"schema\":{\"person\":[\"id\",\"name\",\"name_zh\",\"avatar\",\"num_view\",\"is_follow\",\"work\",\"hide\",\"nation\",\"language\",\"bind\",\"acm_citations\"," +
                "\"links\",\"educations\",\"tags\",\"tags_zh\",\"num_view\",\"num_follow\",\"is_upvoted\",\"num_upvoted\",\"is_downvoted\",\"is_lock\"," +
                "{\"indices\":[\"hindex\",\"pubs\",\"citations\"]}," +
                "{\"profile\":[\"position\",\"position_zh\",\"affiliation\",\"affiliation_zh\",\"work\",\"gender\",\"lang\",\"homepage\",\"phone\",\"email\",\"fax\"," +
                "\"bio\",\"bio_zh\",\"edu\",\"address\",\"note\",\"homepage\",\"titles\"]}]}}]";
        try {
            Thread.sleep(CommonUtils.randomRange(100, 500));
            okhttp3.Response response = OkHttp3Util.doPostJSON(Constants.AMINER_PERSON_INFO_PATH, requestPathStr, 5000, 3, false);
            String result = response.body().string();
            if (StringUtils.isBlank(result))
                return;
            result = CommonUtils.jsonUnicodeToUtf8(result);
            if (StringUtils.isNotBlank(result)) {
                JSONObject resultJson = JSON.parseObject(result);
                if (resultJson.containsKey("data") && resultJson.getJSONArray("data") != null && resultJson.getJSONArray("data").size() > 0) {
                    JSONArray dataArray = resultJson.getJSONArray("data");
                    JSONObject jsonObject = dataArray.getJSONObject(0);
                    if (null != jsonObject && jsonObject.getBoolean("succeed")) {
                        JSONArray dataArrayTwoLevel = jsonObject.getJSONArray("data");
                        for (int i = 0; i < dataArrayTwoLevel.size(); i++) {
                            JSONObject jsonObjectTwoLevel = dataArrayTwoLevel.getJSONObject(i);
                            String id = jsonObjectTwoLevel.containsKey("id") ? jsonObjectTwoLevel.getString("id") : null;
                            JSONObject linkJsonObject = jsonObjectTwoLevel.containsKey("links") ? jsonObjectTwoLevel.getJSONObject("links") : null;
                            JSONObject profileJsonObject = jsonObjectTwoLevel.containsKey("profile") ? jsonObjectTwoLevel.getJSONObject("profile") : null;
                            String bio = null;
                            String workCompany = null;
                            String homepage = null;
                            String edu = null;
                            String work = null;
                            String positionZh = null;
                            String sex = null;
                            List<Resume> resumeList = new ArrayList<>();
                            JSONArray linkJSONArray = new JSONArray();
                            if (null != profileJsonObject) {
                                // 个人简历
                                bio = profileJsonObject.containsKey("bio") ? profileJsonObject.getString("bio") : null;
                                workCompany = profileJsonObject.containsKey("affiliation") ? profileJsonObject.getString("affiliation") : null;
                                homepage = profileJsonObject.containsKey("homepage") ? profileJsonObject.getString("homepage") : null;
                                // 学习经历
                                edu = profileJsonObject.containsKey("edu") ? profileJsonObject.getString("edu") : null;
                                edu = RegexUtils.replaceData(edu, "\\n", "<br>");
                                // 工作经历
                                work = profileJsonObject.containsKey("work") ? profileJsonObject.getString("work") : null;
                                work = RegexUtils.replaceData(work, "\\n", "<br>");
                                // 职称
                                positionZh = profileJsonObject.containsKey("position_zh") ? profileJsonObject.getString("position_zh") : null;
                                // 性别，male：男，female：女
                                String gender = profileJsonObject.containsKey("gender") ? profileJsonObject.getString("gender") : null;
                                if (StringUtils.equals(gender, "male")) {
                                    sex = "男";
                                } else if (StringUtils.equals(gender, "female")) {
                                    sex = "女";
                                }
                                handlerResume(edu, resumeList);
                                handlerResume(work, resumeList);
                            }

                            if (MapUtils.isNotEmpty(aminerPersonMap) && aminerPersonMap.containsKey(id)) {
                                PersonInfo personInfo = aminerPersonMap.get(id);
                                Integer stamp = personInfo.getStamp();
                                if (null == stamp) {
                                    if (StringUtils.isNotBlank(bio))
                                        personInfo.setDescribeData(bio);
                                    if (CollectionUtils.isNotEmpty(resumeList))
                                        personInfo.setResumeList(resumeList);
                                    if (StringUtils.isNotEmpty(sex))
                                        personInfo.setSex(sex);
                                    if (StringUtils.isNotEmpty(positionZh))
                                        personInfo.setPersonTitle(positionZh);
                                    if (null != linkJsonObject && linkJsonObject.containsKey("resource")) {
                                        JSONArray resourceLinkJSONArray = linkJsonObject.getJSONObject("resource").getJSONArray("resource_link");
                                        for (int j = 0; j < resourceLinkJSONArray.size(); j++) {
                                            JSONObject obj = new JSONObject();
                                            String resourceLinkUrl = resourceLinkJSONArray.getJSONObject(i).getString("url");
                                            if (StringUtils.isNotBlank(resourceLinkUrl)) {
                                                String resourceLinkName = resourceLinkJSONArray.getJSONObject(i).getString("id");
                                                obj.put("relatedLinksName", resourceLinkName);
                                                obj.put("relatedLinksUrl", resourceLinkUrl);
                                                linkJSONArray.add(obj);
                                            }
                                        }
                                    }
                                    if (StringUtils.isNotBlank(homepage)) {
                                        JSONObject obj = new JSONObject();
                                        obj.put("relatedLinksName", "homepage");
                                        obj.put("relatedLinksUrl", homepage);
                                        linkJSONArray.add(obj);
                                    }

                                    if (null != linkJSONArray && linkJSONArray.size() > 0) {
                                        personInfo.setRelatedLinks(JSON.toJSONString(linkJSONArray));
                                    }

                                }
                            }
                        }
                    }
                }
            }
        } catch (InterruptedException e) {
            log.error("aminerPersonInfoData - aminer-个人信息查询，Exception：{}", e);
        } catch (IOException e) {
            log.error("aminerPersonInfoData - aminer-个人信息查询，Exception：{}", e);
        }
    }


    /**
     * 根据aminerId查询论文信息
     *
     * @param paramsMap       查询条件
     * @param aminerPersonMap
     */
    public void getPaperDataByAminerId(Map<String, Object> paramsMap, Map<String, PersonInfo> aminerPersonMap) {
        String requestPathStr = "[{\"action\":\"person.GetPersonPubs\"," +
                "\"parameters\":{\"offset\":$${1}," +
                "\"size\":$${2},\"sorts\":[\"!year\"]," +
                "\"ids\":[\"$${3}\"],\"searchType\":\"all\"}," +
                "\"schema\":{\"publication\":[\"id\",\"year\",\"title\",\"title_zh\",\"authors._id\",\"authors.name\",\"authors.name_zh\",\"num_citation\",\"venue.info.name\"," +
                "\"venue.volume\",\"venue.info.name_zh\",\"venue.issue\",\"pages.start\",\"pages.end\",\"lang\",\"pdf\",\"doi\",\"urls\",\"versions\"]}}]";
        try {
            Thread.sleep(CommonUtils.randomRange(100, 500));
            Integer size = (Integer) paramsMap.get("pageSize");
            Integer offset = (Integer) paramsMap.get("pageOffset");
            String aminerPaperPath = RegexUtils.replaceData(requestPathStr, "\\$\\$\\{1}", paramsMap.get("pageOffset").toString());
            aminerPaperPath = RegexUtils.replaceData(aminerPaperPath, "\\$\\$\\{2}", paramsMap.get("pageSize").toString());
            aminerPaperPath = RegexUtils.replaceData(aminerPaperPath, "\\$\\$\\{3}", paramsMap.get("aminerPersonId").toString());
            okhttp3.Response response = OkHttp3Util.doPostJSON(Constants.AMINER_PAPER_BY_AMINER_ID_PATH, aminerPaperPath, 5000, 3, false);
            String result = response.body().string();
            if (StringUtils.isBlank(result))
                return;
            result = CommonUtils.jsonUnicodeToUtf8(result);
            if (StringUtils.isNotBlank(result)) {
                JSONObject resultJson = JSON.parseObject(result);
                if (resultJson.containsKey("data") && resultJson.getJSONArray("data") != null && resultJson.getJSONArray("data").size() > 0) {
                    JSONArray dataArray = resultJson.getJSONArray("data");
                    JSONObject jsonObject = dataArray.getJSONObject(0);
                    if (null != jsonObject && jsonObject.getBoolean("succeed")) {
                        JSONArray paperItems = jsonObject.containsKey("items") ? jsonObject.getJSONArray("items") : null;
                        if (null != paperItems && paperItems.size() > 0 && aminerPersonMap.containsKey(paramsMap.get("aminerPersonId").toString())) {
                            PersonInfo personInfo = aminerPersonMap.get(paramsMap.get("aminerPersonId").toString());
                            parseAminerPaperItems(paperItems, personInfo);
                        }

                        JSONObject keyValuesJsonObject = jsonObject.containsKey("keyValues") ? jsonObject.getJSONObject("keyValues") : null;
                        if (null != keyValuesJsonObject) {
                            Integer total = keyValuesJsonObject.getInteger("total");
                            // 计算总页数
                            Integer allPageSize = CommonUtils.getPages(total, size);

                            // 查询分页数据，从第二页开始
                            for (int i = 2; i <= allPageSize; i++) {
                                Integer newPageOffset = size + offset;
                                aminerPaperPath = RegexUtils.replaceData(requestPathStr, "\\$\\$\\{1}", String.valueOf(newPageOffset));
                                aminerPaperPath = RegexUtils.replaceData(aminerPaperPath, "\\$\\$\\{2}", paramsMap.get("pageSize").toString());
                                aminerPaperPath = RegexUtils.replaceData(aminerPaperPath, "\\$\\$\\{3}", paramsMap.get("aminerPersonId").toString());

                                Thread.sleep(CommonUtils.randomRange(100, 500));
                                response = OkHttp3Util.doPostJSON(Constants.AMINER_JQ_PERSONS_LIST_PATH, aminerPaperPath, 5000, 3, false);
                                result = response.body().string();
                                if (StringUtils.isBlank(result))
                                    continue;
                                result = CommonUtils.jsonUnicodeToUtf8(result);
                                resultJson = JSON.parseObject(result);
                                if (resultJson.containsKey("data") && resultJson.getJSONArray("data") != null && resultJson.getJSONArray("data").size() > 0) {
                                    dataArray = resultJson.getJSONArray("data");
                                    jsonObject = dataArray.getJSONObject(0);
                                    if (null != jsonObject && jsonObject.getBoolean("succeed")) {
                                        paperItems = jsonObject.containsKey("items") ? jsonObject.getJSONArray("items") : null;
                                        if (null != paperItems && paperItems.size() > 0) {
                                            PersonInfo personInfo = aminerPersonMap.get(paramsMap.get("aminerPersonId").toString());
                                            parseAminerPaperItems(paperItems, personInfo);
                                        }
                                    }
                                }
                            }
                        }
                    }
                }
            }
        } catch (InterruptedException e) {
            log.error("getPaperDataByAminerId - 根据aminerId查询论文信息异常，Exception：{}", e);
        } catch (IOException e) {
            log.error("getPaperDataByAminerId - 根据aminerId查询论文信息异常，Exception：{}", e);
        }
    }

    /**
     * 解析全球华人库-论文列表
     *
     * @param paperItems
     */
    public void parseAminerPaperItems(JSONArray paperItems, PersonInfo personInfo) {
        List<Paper> paperList = new ArrayList<>();
        for (int i = 0; i < paperItems.size(); i++) {
            JSONObject jsonObject = paperItems.getJSONObject(i);

            Paper paper = parsePaperData(jsonObject);
            paper.setPersonId(personInfo.getId());

            if (null != paper && StringUtils.isNotBlank(paper.getTitle()) && StringUtils.isNotBlank(paper.getPublicationDate()))
                paperList.add(paper);
        }

        if (CollectionUtils.isNotEmpty(paperList)) {
            if (CollectionUtils.isNotEmpty(personInfo.getPaperList())) {
                List<Paper> personInfoPaperList = personInfo.getPaperList();
                personInfoPaperList.addAll(paperList);
            } else {
                personInfo.setPaperList(paperList);
            }
        }
    }

    public Paper parsePaperData(JSONObject jsonObject) {
        Paper paper = new Paper();
        JSONArray authorsJSONArray = jsonObject.containsKey("authors") ? jsonObject.getJSONArray("authors") : null;
        String abstractData = jsonObject.containsKey("abstract") ? jsonObject.getString("abstract") : null;
        String doi = jsonObject.containsKey("doi") ? jsonObject.getString("doi") : null;
        // 语言，en：外文，zh：中文
        String lang = jsonObject.containsKey("lang") ? jsonObject.getString("lang") : null;
        // 标题
        String title = jsonObject.containsKey("title") ? jsonObject.getString("title") : null;
        // 发表日期
        String publicationDate = jsonObject.containsKey("year") ? jsonObject.getString("year") : null;
        JSONObject venueJSONObject = jsonObject.containsKey("venue") ? jsonObject.getJSONObject("venue") : null;
        if (null != venueJSONObject && venueJSONObject.size() > 0) {
            JSONObject venueInfoJSONObject = venueJSONObject.containsKey("info") ? venueJSONObject.getJSONObject("info") : null;
            if (null != venueInfoJSONObject && venueInfoJSONObject.size() > 0) {
                String periodicalName = venueInfoJSONObject.containsKey("name") ? venueInfoJSONObject.getString("name") : null;
                String publisher = venueInfoJSONObject.containsKey("publisher") ? venueInfoJSONObject.getString("publisher") : null;
                paper.setPeriodical(periodicalName);
                paper.setPublisher(publisher);
            }
        }

        if (null != authorsJSONArray && authorsJSONArray.size() > 0) {
            List<String> authorList = new ArrayList<>();
            for (Object authorObj : authorsJSONArray) {
                JSONObject authorJson = (JSONObject) authorObj;
                String authorName = authorJson.containsKey("name") ? authorJson.getString("name") : null;
                authorList.add(authorName);
            }
            if (CollectionUtils.isNotEmpty(authorList)) {
                paper.setOtherAuthor(StringUtils.join(authorList, ","));
                paper.setFirstAuthor(authorList.get(0));
            }
        }
        if (StringUtils.equalsIgnoreCase(lang, "en")) {
            // 外文
            paper.setLanguageStandard(1);
        } else if (StringUtils.equalsIgnoreCase(lang, "zh")) {
            // 中文
            paper.setLanguageStandard(0);
        }
        if (StringUtils.isNotBlank(doi))
            paper.setDoi(doi);
        if (StringUtils.isNotBlank(title))
            paper.setTitle(title);
        if (StringUtils.isNotBlank(publicationDate))
            paper.setPublicationDate(publicationDate);
        if (StringUtils.isNotBlank(abstractData))
            paper.setDescribeData(abstractData);

        return paper;
    }

    public void handlerResume(String data, List<Resume> resumeList) {
        if (data.indexOf("<br>") > 0) {
            String[] edus = data.split("<br>");
            for (int k = 0; k < edus.length; k++) {
                if (StringUtils.isNotBlank(edus[k])) {
                    Resume resume = new Resume().builder()
                            .resume(edus[k])
                            .build();
                    resumeList.add(resume);
                }
            }
        } else {
            Resume resume = new Resume().builder()
                    .resume(data)
                    .build();
            resumeList.add(resume);
        }
    }

    public void youthChangJiangScholar() {
        try {
//            List<PersonInfo> list = scholarService.parseScholarExcel("C:\\Users\\user\\Documents\\youth_cj_scholar.xlsx");
            List<PersonInfo> list = new ScholarService().parseScholarExcel("C:\\Users\\user\\Documents\\youth_cj_scholar.xlsx");
            Map<String, Long> repeatNameCountMap = list.stream().collect(Collectors.groupingBy(PersonInfo::getName, Collectors.counting()));
            List<String> repeatNameList = repeatNameCountMap.entrySet().stream().filter(entry -> entry.getValue() > 1).map(entry -> entry.getKey()).collect(Collectors.toList());
            if (CollectionUtils.isNotEmpty(repeatNameList)) {
//                System.out.println(JSON.toJSONString(repeatNameList));
                return;
            }
            Map<String, PersonInfo> perosnInfoMap = list.stream().filter(p -> StringUtils.isNotBlank(p.getName())).collect(Collectors.toMap(PersonInfo::getName, (perosnInfo) -> perosnInfo));
            Map<String, Object> personInfoParamsMap = new HashMap<>();
            personInfoParamsMap.put("pageOffset", 0);
            personInfoParamsMap.put("pageSize", 20);
            personInfoParamsMap.put("listId", "5d5a476d530c70b992520a31");
//            for (Map.Entry<String, PersonInfo> entry : perosnInfoMap.entrySet()) {
//                if (StringUtils.equalsAny(entry.getKey(), "杜亚楠")) {
//                    personInfoParamsMap.put("queryByName", entry.getKey());
//                    new JQCrawlerService().aminerPersonInfoList(personInfoParamsMap, perosnInfoMap);
//                    List<PersonInfo> personInfoList = new ArrayList<>();
//                    personInfoList.add(perosnInfoMap.get(entry.getKey()));
//                    new PersonInfoServiceImpl().writeExcel("C:\\Users\\user\\Documents\\", "youth_cj_scholar-02.xlsx", personInfoList);
//                }
//            }


            List<String> resultList = new JQCrawlerService().aminerPersonInfoList(personInfoParamsMap, perosnInfoMap);
            if (CollectionUtils.isNotEmpty(resultList)) {
                FileUtils.writeLines(new File("C:\\Users\\user\\Documents\\youth-1996.txt"), StandardCharsets.UTF_8.toString(), resultList);
            }


            List<String> readList = FileUtils.readLines(new File("C:\\Users\\user\\Documents\\youth-1996.txt"), StandardCharsets.UTF_8);
            for (String line : readList) {
//                new JQCrawlerService().parseAminerTxt(line, perosnInfoMap, null);
            }
            List<PersonInfo> personInfoList = perosnInfoMap.entrySet().stream().map(entry -> entry.getValue()).collect(Collectors.toList());
            List<String> nameList = personInfoList.stream().map(personInfo -> personInfo.getName()).collect(Collectors.toList());

//            new PersonInfoServiceImpl().writeExcel("C:\\Users\\user\\Documents\\", "youth_cj_scholar-03.xlsx", personInfoList);
        } catch (IOException e) {
            e.printStackTrace();
        }

    }

    public static void main(String[] args) throws IOException {
//        List<String> list = new ArrayList<>();
//        list.add("1");
//        list.add("2");
//        list.add("3");
//        list.add("4");
//        list.add("5");
//        FileUtils.writeLines(new File("C:\\Users\\user\\Documents\\1.txt"), StandardCharsets.UTF_8.toString(), list);
//        new JQCrawlerService().youthChangJiangScholar();

//        Map<String, PersonInfo> perosnInfoMap = new HashMap<>();
//        List<String> readList = FileUtils.readLines(new File("E:\\var\\server\\crawler\\youth-1996.txt"), StandardCharsets.UTF_8);
//        for (String line : readList) {
//            parseAminerTxt(line, perosnInfoMap, null);
//        }
//        List<PersonInfo> personInfoList = perosnInfoMap.entrySet().stream().map(entry -> entry.getValue()).collect(Collectors.toList());
//        List<String> nameList = personInfoList.stream().map(personInfo -> personInfo.getName()).collect(Collectors.toList());
//        log.warn("youth 年度：{}，总条数:{}，人员姓名：{}", 1996, nameList.size(), JSON.toJSONString(nameList));
//        new JQCrawlerService().youth();


//        List<PersonInfo> list = new JQCrawlerService().crawlerOutstanTalents();
//        Map<String, List<PersonInfo>> personInfoGroupMap = list.stream().collect(Collectors.groupingBy(PersonInfo::getElectionYear));
////        System.out.println(JSON.toJSONString(personInfoGroupMap));
//        // 全球华人专家库-杰青年度列表，personInfoParamsList里listId为杰青列表id,year为年度
//        List<Map<String, Object>> personInfoParamsList = new JQCrawlerService().aminerList();
////        // {"2006":"5d5a4741530c70b9925209f5","2016":"57a837809ed5dba7786c3f9f","2005":"5d5a4745530c70b9925209fb",
//// "2003":"5d5a474c530c70b992520a07","2004":"5d5a4749530c70b992520a00","2001":"5d5a4754530c70b992520a10","2002":"5d5a4751530c70b992520a0d",
//// "1999":"5d5a475d530c70b992520a1b","2000":"5d5a4758530c70b992520a16","1997":"5d5a4764530c70b992520a21","1998":"5d5a4760530c70b992520a1f",
//// "1996":"5d5a4769530c70b992520a27","1995":"5d5a476d530c70b992520a31","1994":"5d5a4771530c70b992520a35","2017":"5d50bf76530c70a9b362adab",
//// "2020":"5f327cc24c775e3a7964f33d","2019":"5d43870e530c7031c32bdf3a","2018":"5d50bed3530c70a9b362ad9f","2015":"5d525f7a530c70a9b3631ecf",
//// "2013":"5d53a90e530c70a9b363721b","2010":"5d53a91e530c70a9b3637225","2008":"5d5a4738530c70b9925209e9","2009":"5d53a922530c70a9b3637227",
//// "2007":"5d5a473d530c70b9925209ef","2012":"5d53a916530c70a9b3637221","2011":"5d53a919530c70a9b3637223","2014":"59267feb9ed5db8c189eefea"}
//
//        if (CollectionUtils.isNotEmpty(personInfoParamsList)) {
//            for (int i = 0; i < personInfoParamsList.size(); i++) {
//                Map<String, Object> personInfoParamsMap = personInfoParamsList.get(i);
//                personInfoParamsMap.put("pageOffset", 0);
//                personInfoParamsMap.put("pageSize", 20);
//                personInfoParamsMap.forEach((key, yearValue) -> {
//                    if (StringUtils.equals(key, "year") && personInfoGroupMap.containsKey(yearValue) && CollectionUtils.isNotEmpty(personInfoGroupMap.get(yearValue))) {
//                        List<PersonInfo> personInfoList = personInfoGroupMap.get(yearValue);
//
//                        Map<String, PersonInfo> nameMap = new HashMap<>();
//                        // 同年重名人员，2019:石峰、张龙，2014:张强，2009:汪国平，2007：王伟，暂不处理，也没有几个手动解决
//                        if (StringUtils.equals(yearValue.toString(), "2019")) {
//                            nameMap = personInfoList.stream().filter(personInfo -> null == personInfo.getStamp()
//                                    && !StringUtils.equalsAny(personInfo.getName(), "石峰", "张龙"))
//                                    .collect(Collectors.toMap(PersonInfo::getName, x -> x));
//                        } else if (StringUtils.equals(yearValue.toString(), "2014")) {
//                            nameMap = personInfoList.stream().filter(personInfo -> null == personInfo.getStamp()
//                                    && !StringUtils.equalsAny(personInfo.getName(), "张强"))
//                                    .collect(Collectors.toMap(PersonInfo::getName, x -> x));
//                        } else if (StringUtils.equals(yearValue.toString(), "2009")) {
//                            nameMap = personInfoList.stream().filter(personInfo -> null == personInfo.getStamp()
//                                    && !StringUtils.equalsAny(personInfo.getName(), "汪国平"))
//                                    .collect(Collectors.toMap(PersonInfo::getName, x -> x));
//                        } else if (StringUtils.equals(yearValue.toString(), "2007")) {
//                            nameMap = personInfoList.stream().filter(personInfo -> null == personInfo.getStamp()
//                                    && !StringUtils.equalsAny(personInfo.getName(), "王伟"))
//                                    .collect(Collectors.toMap(PersonInfo::getName, x -> x));
//                        } else {
//                            nameMap = personInfoList.stream().filter(personInfo -> null == personInfo.getStamp())
//                                    .collect(Collectors.toMap(PersonInfo::getName, x -> x));
//                        }
//
//                        System.out.println(yearValue + "年，" + nameMap.size());
////                        // 人员信息列表
//                        new JQCrawlerService().aminerPersonInfoList(personInfoParamsMap, nameMap);
//                    }
//                });
//            }
//        }
//
////        System.out.println(list.size());
//        System.out.println(personInfoGroupMap.size());


//        personInfoGroupMap.forEach((key, value) -> {
////            List<PersonInfo> aminerIdPersonInfoEmptyList = value.stream().filter(personInfo -> StringUtils.isBlank(personInfo.getAminerId())).collect(Collectors.toList());
////            log.info("aminerId为空的数据：{}", JSON.toJSONString(aminerIdPersonInfoEmptyList));
//            Map<String, PersonInfo> aminerPersonMap = value.stream().filter(personInfo -> StringUtils.isNotBlank(personInfo.getAminerId())).collect(Collectors.toMap(PersonInfo::getAminerId, x -> x));
//            int allSize = value.size();
//            int size = 10;
//            Integer allPageSize = CommonUtils.getPages(allSize, size);
//            for (int i = 0; i < allPageSize; i++) {
//                List<PersonInfo> subPersonInfoList = new ArrayList<>();
//                if ((i * size + size) > value.size()) {
//                    subPersonInfoList = value.subList(i * size, value.size());
//                } else {
//                    subPersonInfoList = value.subList(i * size, i * size + size);
//                }
//                List<String> aminerIdList = subPersonInfoList.stream().filter(personInfo -> (null == personInfo.getStamp() && StringUtils.isNotBlank(personInfo.getAminerId())))
//                        .map(personInfo -> personInfo.getAminerId()).collect(Collectors.toList());
//                if (CollectionUtils.isNotEmpty(aminerIdList)) {
//                    for (int j = aminerIdList.size() - 1; j >= 0; j--) {
//                        if (StringUtils.isBlank(aminerIdList.get(j))) {
//                            aminerIdList.remove(j);
//                        }
//                    }
//                    System.out.println(StringUtils.join(aminerIdList, ","));
////                    new JQCrawlerService().aminerPersonInfoData(StringUtils.join(aminerIdList, ","), aminerPersonMap);
//
//                    StringBuilder sb = new StringBuilder();
//                    for (int j = 0; j < aminerIdList.size(); j++) {
//                        if (StringUtils.isNotBlank(aminerIdList.get(j))) {
//                            Map<String, Object> paramsMap = new HashMap<>();
//                            paramsMap.put("pageOffset", 0);
//                            paramsMap.put("pageSize", 20);
//                            paramsMap.put("aminerPersonId", aminerIdList.get(j));
//                            new JQCrawlerService().getPaperDataByAminerId(paramsMap, aminerPersonMap);
//                            if (j == aminerIdList.size() - 1) {
//                                sb.append("\"").append(aminerIdList.get(j)).append("\"");
//                            } else {
//                                sb.append("\"").append(aminerIdList.get(j)).append("\"").append(",");
//                            }
//                        }
//                    }
//                    new JQCrawlerService().aminerPersonInfoData(sb.toString(), aminerPersonMap);
//                }
//
//            }
//        });

    }
}
